In [24]:
import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
import plotnine as gg
#matplotlib.style.use('ggplot')
matplotlib.style.use('seaborn')
matplotlib.rcParams['figure.figsize'] = (12,8)
%matplotlib inline
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import scale
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix
from sklearn.cluster import KMeans
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_curve,auc
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score
import datetime
import time
In [194]:
# calculate the score of model
def score(fpr,tpr):
score=0.4*tpr[np.where(fpr>=0.001)[0][0]]+0.3*tpr[np.where(fpr>=0.005)[0][0]]+0.3*tpr[np.where(fpr>=0.01)[0][0]]
return score
In [202]:
# draw roc and pr curve
def roc_and_pr(y_te, y_score):
fpr, tpr, threshold = roc_curve(y_te, y_score)
roc_auc = auc(fpr, tpr)
lw = 2
plt.figure(figsize = (15,8))
plt.subplot(121)
plt.plot(fpr, tpr, color='darkorange', lw=lw, label='ROC curve (area = %0.3f)' % roc_auc)
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC')
plt.legend(loc="lower right")
precision, recal, thresholds = precision_recall_curve(y_te, y_score)
average_precision = average_precision_score(y_te, y_score)
plt.subplot(122)
plt.step(recal, precision, color='b', alpha=0.2,where='post')
plt.fill_between(recal, precision, step='post', alpha=0.2,color='b')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('PR curve: AP={0:0.3f}'.format(average_precision))
In [5]:
data = pd.read_csv('atec_anti_fraud_train.csv')
In [6]:
data.head()
Out[6]:
In [6]:
data.shape
Out[6]:
1. Exploratory Analysis¶
1.1 date¶
In [7]:
date_df = data.iloc[:,1:3]
In [8]:
date_df.head()
Out[8]:
In [9]:
def strptime(date):
t = time.strptime(str(date), "%Y%m%d")
return(t)
data_strptime = date_df.date.apply(strptime)
In [10]:
date_df['year'] = [data_strptime[i][0] for i in range(len(data_strptime))]
date_df['month'] = [data_strptime[i][1] for i in range(len(data_strptime))]
date_df['day'] = [data_strptime[i][2] for i in range(len(data_strptime))]
date_df['weekday'] = [data_strptime[i][6] for i in range(len(data_strptime))]
date_df['strptime'] = data_strptime
In [11]:
date_df.head()
Out[11]:
In [29]:
date_df = date_df.sort_values(by='strptime')
In [30]:
date_df.head()
Out[30]:
In [31]:
date_df.tail()
Out[31]:
* The data is from 2017-09-05 to 2017-11-05.
split the data based on date.¶
train data: date between [ 20170905, 20171020]
test data: date between [ 20171021, 20171105]
Almost 3 : 1.
Store as csv
In [90]:
thres = time.strptime(str(20171020), "%Y%m%d")
train1020_with_unlabel = data[date_df['strptime'] <=thres]
test1020_with_unlabel = data[date_df['strptime'] >thres]
data_without_unlabel = data[data['label'] != -1]
date_ = date_df[date_df['label'] != -1 ]
train1020_without_unlabel = data_without_unlabel[date_['strptime'] <=thres]
test1020_without_unlabel = data_without_unlabel[date_['strptime'] >thres]
In [89]:
train1020_with_unlabel.to_csv('/Users/lijh/Downloads/ATEC/train1020_with_unlabel.csv',index=False)
test1020_with_unlabel.to_csv('/Users/lijh/Downloads/ATEC/test1020_with_unlabel.csv',index=False)
train1020_without_unlabel.to_csv('/Users/lijh/Downloads/ATEC/train1020_only_label.csv',index=False)
test1020_without_unlabel.to_csv('/Users/lijh/Downloads/ATEC/test1020_only_label.csv',index=False)
1.1.1 weekday¶
In [91]:
label_0 = date_df[date_df['label'] == 0]
label_1 = date_df[date_df['label'] == 1]
unlabel = date_df[date_df['label'] == -1]
In [93]:
plt.rcParams['figure.figsize'] = (20,15)
plt.subplot(221)
date_df.weekday.hist(bins = 13, density = True)
plt.title('label = 0', fontsize = 30)
plt.subplot(222)
label_0.weekday.hist(bins = 13, density = True)
plt.title('label = 0', fontsize = 30)
plt.subplot(223)
label_1.weekday.hist(bins = 13, density = True)
plt.title('label = 1', fontsize = 30)
plt.xlabel('weekday',fontsize = 20)
plt.subplot(224)
unlabel.weekday.hist(bins = 13, density =True)
plt.title('label = -1', fontsize = 30)
plt.xlabel('weekday',fontsize = 20)
Out[93]:
In [94]:
label0_weekday = []
label1_weekday = []
unlabel_weekday = []
for i in range(7):
weekday_ = date_df[date_df['weekday']==i]
num = len(weekday_)
label0_weekday.append( len(weekday_[weekday_['label'] == 0]) / num )
label1_weekday.append( len(weekday_[weekday_['label'] == 1]) / num )
unlabel_weekday.append( len(weekday_[weekday_['label'] == -1]) / num )
In [95]:
each_weekday = pd.DataFrame({'label 0': label0_weekday, 'label 1': label1_weekday,
'unlabel': unlabel_weekday})
In [99]:
cf.go_offline()
each_weekday[['label 1','unlabel']].iplot(kind='spread', title='The rate of label = 1 and unlabel in each weekday',yTitle = 'rate')
* Friday, Saturday and Sunday are more likely to having risk.
1.1.2 time in order¶
In [98]:
import pygal
import plotly.plotly as py
import plotly.graph_objs as go
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
init_notebook_mode(connected=True)
from plotly import __version__
import cufflinks as cf
In [100]:
day = date_df['date'].value_counts().index.values
day.sort()
day
Out[100]:
In [101]:
label0_day = []
label1_day = []
unlabel_day = []
for i in day:
day_ = date_df[date_df['date']==i]
num = len(day_)
label0_day.append( len(day_[day_['label'] == 0]) / num )
label1_day.append( len(day_[day_['label'] == 1]) / num )
unlabel_day.append( len(day_[day_['label'] == -1]) / num )
In [102]:
each_day = pd.DataFrame({'day': day, 'label 0': label0_day, 'label 1': label1_day,
'unlabel': unlabel_day})
In [103]:
ind = []
for i in range(26):
ind.append( 'Sep ' + str(i+5) )
for i in range(31):
ind.append( 'Oct ' + str(i+1) )
for i in range(5):
ind.append( 'Nov ' + str(i+1) )
In [104]:
each_day.index = ind
In [105]:
cf.go_offline()
each_day[['label 1','unlabel']].iplot(kind='spread', title='The rate of label = 1 and unlabel in each day',yTitle = 'rate')
2. Modeling¶
In [7]:
X_raw = data.iloc[:,3:]
y_raw = data.iloc[:,1]
In [146]:
pca = PCA(n_components = 2)
In [41]:
X_minus = X_raw.copy()
X_minus[X_minus.isna()] = -1
X_label_1 = X_minus[y_raw != -1]
y_label = y_raw[y_raw != -1]
In [15]:
rnd_clf_9 = RandomForestClassifier(n_estimators=100, max_leaf_nodes=20, n_jobs = -1, random_state = 7,
class_weight = 'balanced')
rnd_clf_9.fit(X_label_1, y_label)
Out[15]:
In [17]:
df_9 = pd.DataFrame({'feature': X_label_1.columns,
'importance': rnd_clf_9.feature_importances_ })
df_9 = df_9.sort_values(by='importance', ascending =False)
df_9_ = df_9[df_9['importance'] != 0 ]
print(str(len(df_9_)) + ' features importance != 0')
sns.set(rc={'figure.figsize':(10,10)})
sns.set(font_scale=2) # set the font size
ax = sns.barplot(y = 'feature', x = 'importance',data=df_9[df_9['importance'] >= 0.01 ]) #draw the barplot
ax.set(title = " ", xlabel = "feature", ylabel = "importance")
ax.set_xticklabels(ax.get_xticklabels(), rotation = 90, fontsize = 0.01)
ax.tick_params(labelsize=20)
2.1.2 Impute by median¶
In [25]:
X = X_raw.copy()
imputer = Imputer(strategy = 'median')
imputer.fit(X)
X_norm = imputer.transform(X)
X = pd.DataFrame(X_norm, columns = X.columns)
2.2 Random forest for feature importance¶
- We use labeled data first.
In [26]:
X_label = X[y_raw != -1]
X_label.shape
Out[26]:
In [27]:
y_label = y_raw[y_raw != -1]
y_label.shape
Out[27]:
- Using three different methods for unbalanced problem.
2.2.1 class_weight = 'balanced_subsample'¶
In [28]:
rnd_clf_1 = RandomForestClassifier(n_estimators=100, max_leaf_nodes=20, n_jobs = -1, random_state = 7,
class_weight = 'balanced_subsample')
rnd_clf_1.fit(X_label, y_label)
Out[28]:
In [36]:
df_1 = pd.DataFrame({'feature': X_label.columns,
'importance': rnd_clf_1.feature_importances_ })
df_1 = df_1.sort_values(by='importance', ascending =False)
df_1_ = df_1[df_1['importance'] != 0 ]
print(str(len(df_1_)) + ' features importance != 0')
sns.set(rc={'figure.figsize':(10,10)})
sns.set(font_scale=2) # set the font size
ax = sns.barplot(y = 'feature', x = 'importance',data=df_1[df_1['importance'] >= 0.01 ]) #draw the barplot
ax.set(title = " ", xlabel = "feature", ylabel = "importance")
ax.set_xticklabels(ax.get_xticklabels(), rotation = 90, fontsize = 0.01)
ax.tick_params(labelsize=20)
2.2.3 class_weight = 'balanced'¶
In [37]:
rnd_clf_2 = RandomForestClassifier(n_estimators=100, max_leaf_nodes=20, n_jobs = -1, random_state = 7,
class_weight = 'balanced')
rnd_clf_2.fit(X_label, y_label)
Out[37]:
In [40]:
df_2 = pd.DataFrame({'feature': X_label.columns,
'importance': rnd_clf_2.feature_importances_ })
df_2 = df_2.sort_values(by='importance', ascending =False)
df_2_ = df_2[df_2['importance'] != 0 ]
print(str(len(df_2_)) + ' features importance != 0')
sns.set(rc={'figure.figsize':(10,10)})
sns.set(font_scale=2) # set the font size
ax = sns.barplot(y = 'feature', x = 'importance', data=df_2[df_2['importance'] >= 0.01] ) #draw the barplot
ax.set(title = " ", xlabel = "feature", ylabel = "importance")
ax.set_xticklabels(ax.get_xticklabels(), rotation = 90, fontsize = 0.01)
ax.tick_params(labelsize=20)
In [41]:
t = pd.concat([df_1_, df_2_], axis = 1)
diff = []
for i in range(len(t)):
if True in t.isnull().values[i]:
diff.append(i)
t.iloc[diff,:]
Out[41]:
- The different for these two methods are whether f56 and f190 are in the model or not.
2.2.3 fit( sample_weight )¶
In [42]:
num1 = sum(y_label == 1)
num0 = sum(y_label == 0)
print("label=1: " + str(num1) + "\nlabel=0: " + str(num0)+
"\nratio: " + str( num0 / num1 ))
2.2.3.1 weight: 80 : 1¶
In [43]:
weight_1 = np.ones(len(y_label))
weight_1[y_label == 0] = 1 / ( num1*80 + num0 )
weight_1[y_label == 1] = 80 / ( num1*80 + num0 )
sum(weight_1)
Out[43]:
In [44]:
rnd_clf_3 = RandomForestClassifier(n_estimators=100, max_leaf_nodes=20, n_jobs = -1, random_state = 7)
rnd_clf_3.fit(X_label, y_label, sample_weight = weight_1)
Out[44]:
In [45]:
df_3 = pd.DataFrame({'feature': X_label.columns,
'importance': rnd_clf_3.feature_importances_ })
df_3 = df_3.sort_values(by='importance', ascending =False)
df_3_ = df_3[df_3['importance'] != 0 ]
print(str(len(df_3_)) + ' features importance != 0')
sns.set(rc={'figure.figsize':(10,10)})
sns.set(font_scale=2) # set the font size
ax = sns.barplot(y = 'feature', x = 'importance', data=df_3[df_3['importance'] >= 0.01] ) #draw the barplot
ax.set(title = " ", xlabel = "feature", ylabel = "importance")
ax.set_xticklabels(ax.get_xticklabels(), rotation = 90, fontsize = 0.01)
ax.tick_params(labelsize=20)
In [46]:
t2 = pd.concat([df_1_, df_2_, df_3_], axis = 1)
diff = []
for i in range(len(t)):
if True in t2.isnull().values[i]:
diff.append(i)
t2.iloc[diff,:]
Out[46]:
2.2.3.1 weight : 30 : 1¶
In [47]:
weight_2 = np.ones(len(y_label))
weight_2[y_label == 0] = 1 / ( num1*30 + num0 )
weight_2[y_label == 1] = 30 / ( num1*30 + num0 )
sum(weight_2)
Out[47]:
In [48]:
rnd_clf_4 = RandomForestClassifier(n_estimators=100, max_leaf_nodes=20, n_jobs = -1, random_state = 7)
rnd_clf_4.fit(X_label, y_label, sample_weight = weight_2)
Out[48]:
In [51]:
df_4 = pd.DataFrame({'feature': X_label.columns,
'importance': rnd_clf_4.feature_importances_ })
df_4 = df_4.sort_values(by='importance', ascending =False)
df_4_ = df_4[df_4['importance'] != 0 ]
print(str(len(df_4_)) + ' features importance != 0')
sns.set(rc={'figure.figsize':(10,10)})
sns.set(font_scale=2) # set the font size
ax = sns.barplot(y = 'feature', x = 'importance', data=df_4[df_4['importance'] >= 0.01] ) #draw the barplot
ax.set(title = " ", xlabel = "feature", ylabel = "importance")
ax.set_xticklabels(ax.get_xticklabels(), rotation = 90, fontsize = 0.01)
ax.tick_params(labelsize=20)
In [52]:
t3 = pd.concat([df_1_, df_2_, df_3_, df_4_], axis = 1)
diff = []
for i in range(len(t)):
if True in t3.isnull().values[i]:
diff.append(i)
t3.iloc[diff,:].head()
Out[52]:
In [58]:
print("There are " + str(len(t3.iloc[diff,:])) + " differences.")
- Conclusion: balanced_subsample， balanced or the sample_weight = 80:1 are the same. But if unbalancing, the results differ a lot.
In [59]:
pd.concat([df_1, df_2, df_3, df_4], axis = 1).loc[35:46,:]
Out[59]:
- The importance of f36 - f47, which have lots of NA, are all 0。 So we can drop them directly.
2.3 Prediction¶
2.3.1 Without using unlabel data¶
In [60]:
y_tr, y_te = train_test_split(y_label, test_size = 0.2, random_state =42)
print(len(y_tr), "train+", len(y_te), 'test')
In [61]:
X_tr = X.loc[y_tr.index,:]
X_te = X.loc[y_te.index,:]
In [62]:
rnd_clf_5 = RandomForestClassifier(n_estimators=100, max_leaf_nodes=20, n_jobs = -1, random_state = 7,
class_weight = 'balanced')
In [63]:
rnd_clf_5.fit(X_tr, y_tr)
Out[63]:
In [64]:
y_pred_1 = rnd_clf_5.predict(X_te)
In [65]:
confusion_matrix(y_te, y_pred_1)
Out[65]:
In [66]:
y_pred_prob_1 = rnd_clf_5.predict_proba(X_te)
In [67]:
roc_and_pr(y_te, y_pred_prob_1[:,1])
Out[67]:
2.3.2 Set the unlabel value = 1¶
- Since the unlabel is the high risk one, so we set the unlabel value = 1 first.
In [12]:
ind_unlabel = y_raw == -1
In [69]:
X_unlabel = X[ind_unlabel]
In [70]:
X_unlabel.shape
Out[70]:
In [71]:
X_2 = pd.concat([X_tr, X_unlabel])
In [106]:
y_unlabel_1 = pd.DataFrame(np.ones(sum(ind_unlabel)))
In [74]:
y_2 = pd.concat([y_tr, y_unlabel_1]).iloc[:,0]
In [75]:
rnd_clf_6 = RandomForestClassifier(n_estimators=100, max_leaf_nodes=20, n_jobs = -1, random_state = 7,
class_weight = 'balanced')
In [76]:
rnd_clf_6.fit(X_2, y_2)
Out[76]:
In [79]:
y_pred_2 = rnd_clf_6.predict(X_te)
In [80]:
confusion_matrix(y_te, y_pred_2)
Out[80]:
In [81]:
y_pred_prob_2 = rnd_clf_6.predict_proba(X_te)
In [196]:
roc_and_pr(y_te, y_pred_prob_2[:,1])
2.3.2 using the train & test data splited by time split; only labeled data¶
In [219]:
X_tr_1020 = train1020_without_unlabel.iloc[:,3:]
In [220]:
y_tr_1020 = train1020_without_unlabel.iloc[:,1]
In [221]:
X_te_1020 = test1020_without_unlabel.iloc[:,3:]
In [222]:
y_te_1020 = test1020_without_unlabel.iloc[:,1]
In [223]:
X_tr_1020 = X_tr_1020.fillna(-1)
In [224]:
X_te_1020 = X_te_1020.fillna(-1)
In [115]:
rnd_clf_7 = RandomForestClassifier(n_estimators=100, n_jobs = -1, random_state = 7,
class_weight = 'balanced')
In [116]:
rnd_clf_7.fit(X_tr_1020, y_tr_1020)
Out[116]:
In [117]:
y_pred_3 = rnd_clf_7.predict(X_te_1020)
In [118]:
confusion_matrix(y_te_1020, y_pred_3)
Out[118]:
In [119]:
y_pred_prob_3 = rnd_clf_7.predict_proba(X_te_1020)
In [121]:
roc_and_pr(y_te_1020, y_pred_prob_3[:,1])
Out[121]:
In [122]:
score(fpr3, tpr3)
Out[122]:
2.4 肖颂凯的猜想大实验¶
In [165]:
# data_without_unlabel = data[data['label'] != -1]
# date_ = date_df[date_df['label'] != -1 ]
thres1 = time.strptime(str(20170920), "%Y%m%d")
thres2 = time.strptime(str(20171005), "%Y%m%d")
thres3 = time.strptime(str(20171020), "%Y%m%d")
train11 = data_without_unlabel[date_['strptime'] <=thres1]
test11 = data_without_unlabel[ date_['strptime'] > thres3 ]
In [173]:
train22 = data_without_unlabel[(date_['strptime'] >thres1) & (date_['strptime'] <= thres2)]
In [174]:
train33 = data_without_unlabel[(date_['strptime'] >thres2) & (date_['strptime'] <= thres3)]
2.4.1¶
In [166]:
X_tr_11 = train11.iloc[:,3:]
y_tr_11 = train11.iloc[:,1]
X_te_11 = test11.iloc[:,3:]
y_te_11 = test11.iloc[:,1]
X_tr_11 = X_tr_11.fillna(-1)
X_te_11 = X_te_11.fillna(-1)
In [167]:
rnd_clf_8 = RandomForestClassifier(n_estimators=100, n_jobs = -1, random_state = 7,
class_weight = 'balanced')
In [168]:
rnd_clf_8.fit(X_tr_11, y_tr_11)
Out[168]:
In [169]:
y_pred_4 = rnd_clf_8.predict(X_te_11)
In [170]:
confusion_matrix(y_te_11, y_pred_4)
Out[170]:
In [171]:
y_pred_prob_4= rnd_clf_8.predict_proba(X_te_11)
In [204]:
roc_and_pr(y_te_11, y_pred_prob_4[:,1])
In [185]:
score(fpr4, tpr4)
Out[185]:
2.4.2¶
In [180]:
X_tr_22 = train22.iloc[:,3:]
y_tr_22 = train22.iloc[:,1]
X_tr_22 = X_tr_22.fillna(-1)
rnd_clf_10 = RandomForestClassifier(n_estimators=100, n_jobs = -1, random_state = 7,
class_weight = 'balanced')
In [181]:
rnd_clf_10.fit(X_tr_22, y_tr_22)
Out[181]:
In [182]:
y_pred_5 = rnd_clf_10.predict(X_te_11)
confusion_matrix(y_te_11, y_pred_5)
Out[182]:
In [203]:
y_pred_prob_5= rnd_clf_10.predict_proba(X_te_11)
roc_and_pr(y_te_11, y_pred_prob_5[:,1])
In [186]:
score(fpr5, tpr5)
Out[186]:
2.4.3¶
In [188]:
X_tr_33 = train33.iloc[:,3:]
y_tr_33 = train33.iloc[:,1]
X_tr_33 = X_tr_33.fillna(-1)
rnd_clf_11 = RandomForestClassifier(n_estimators=100, n_jobs = -1, random_state = 7,
class_weight = 'balanced')
In [189]:
rnd_clf_11.fit(X_tr_33, y_tr_33)
Out[189]:
In [190]:
y_pred_6 = rnd_clf_11.predict(X_te_11)
confusion_matrix(y_te_11, y_pred_6)
Out[190]:
In [205]:
y_pred_prob_6= rnd_clf_11.predict_proba(X_te_11)
roc_and_pr(y_te_11, y_pred_prob_6[:,1])
In [193]:
score(fpr6,tpr6)
Out[193]:
2.5 姜山的猜想大实验¶
Using the train and the data in 2.3.2, that is, split the train and test set by the date, train : test = 3:1
In [225]:
# rnd_clf_7 = RandomForestClassifier(n_estimators=100, n_jobs = -1, random_state = 7,
# class_weight = 'balanced')
# rnd_clf_7.fit(X_tr_1020, y_tr_1020)
# y_pred_3 = rnd_clf_7.predict(X_te_1020)
In [226]:
confusion_matrix(y_te_1020, y_pred_3)
Out[226]:
In [227]:
y_pred_prob_3 = rnd_clf_7.predict_proba(X_te_1020)
In [228]:
roc_and_pr(y_te_1020, y_pred_prob_3[:,1])
In [243]:
In [20]:
def jiangshan( y_score ):
ind_add = y_score > 0.9
X_add = pd.concat([X_tr_1020, X_te_1020[ind_add]])
print(len(X_add_1) - len(X_tr_1020) )
y_add = pd.concat([y_tr_1020, pd.DataFrame(np.ones(sum(ind_add))) ]).iloc[:,0]
rnd_clf_js = RandomForestClassifier(n_estimators=100, n_jobs = -1, random_state = 7,
class_weight = 'balanced')
rnd_clf_js.fit(X_add, y_add)
y_pred_js = rnd_clf_js.predict(X_te_1020)
print( confusion_matrix(y_te_1020, y_pred_js) )
y_pred_prob_js = rnd_clf_12.predict_proba(X_te_1020)
roc_and_pr(y_te_1020, y_pred_prob_js[:,1])
return(y_pred_prob_js)
In [247]:
jiangshan(y_pred_prob_7[:,1])
Out[247]:
3 test¶
3.1¶
In [8]:
test = pd.read_csv('atec_anti_fraud_test_a.csv')
In [9]:
X_test = test.iloc[:,2:]
In [10]:
X_test_ = X_test.copy()
In [86]:
X_test_norm = imputer.transform(X_test_)
X_test_ = pd.DataFrame(X_test_norm, columns = X_test_.columns)
In [87]:
y_test_prob = rnd_clf_1.predict_proba(X_test_)
score = y_test_prob[:,1]¶
In [89]:
final = pd.DataFrame({'id':test.iloc[:,0], 'score': score})
In [90]:
final.head()
Out[90]:
In [91]:
final.to_csv('/Users/lijh/Downloads/ATEC/final.csv',index=False)
3.2¶
In [108]:
y_3 = y_raw.copy()
In [111]:
y_3[ind_unlabel] = 1
In [113]:
rnd_clf_7 = RandomForestClassifier(n_estimators=100, max_leaf_nodes=20, n_jobs = -1, random_state = 7,
class_weight = 'balanced')
In [115]:
rnd_clf_7.fit(X, y_3)
Out[115]:
In [116]:
y_test_prob_2 = rnd_clf_7.predict_proba(X_test_)
In [117]:
score_2 = y_test_prob_2[:,1]
In [118]:
final_2 = pd.DataFrame({'id':test.iloc[:,0], 'score': score_2})
In [19]:
final_2.head()
In [120]:
final_2.to_csv('/Users/lijh/Downloads/ATEC/final2.0.csv',index=False)
3.3¶
In [130]:
final_3 = pd.DataFrame({'id':test.iloc[:,0]})
In [131]:
final_3['score'] = 1
In [136]:
final_3.head()
Out[136]:
In [132]:
final_3.to_csv('/Users/lijh/Downloads/ATEC/final3.0.csv',index=False)
3.4¶
In [133]:
final_4 = pd.DataFrame({'id':test.iloc[:,0]})
In [134]:
final_4['score'] = 0
In [137]:
final_4.head()
Out[137]:
In [135]:
final_4.to_csv('/Users/lijh/Downloads/ATEC/final4.0.csv',index=False)
3.5¶
In [13]:
y_5 = y_raw.copy()
y_5[ind_unlabel] = 1
In [14]:
X_5 = X_raw.copy()
In [15]:
X_5 = X_5.fillna(-1)
In [16]:
X_test_5 = X_test.copy()
In [19]:
X_test_5 = X_test_5.fillna(-1)
In [28]:
rnd_clf_20 = RandomForestClassifier(n_estimators=100, n_jobs = -1, random_state = 7,
class_weight = 'balanced')
In [30]:
rnd_clf_20.fit(X_5, y_5)
Out[30]:
In [31]:
y_test_prob_5 = rnd_clf_20.predict_proba(X_test_5)
score_5 = y_test_prob_5[:,1]
In [40]:
def jiangshan2( y_score ):
ind_add = y_score > 0.7
X_add = pd.concat([X_5, X_test_5[ind_add]])
print(len(X_add) - len(X_5) )
y_add = pd.concat([y_5, pd.DataFrame(np.ones(sum(ind_add))) ]).iloc[:,0]
rnd_clf_js = RandomForestClassifier(n_estimators=100, n_jobs = -1, random_state = 7,
class_weight = 'balanced')
rnd_clf_js.fit(X_add, y_add)
y_pred_prob_js = rnd_clf_js.predict_proba(X_test_5)
return(y_pred_prob_js)
In [37]:
score_5 = jiangshan2(score_5)[:,1]
In [38]:
score_5 = jiangshan2(score_5)[:,1]
In [39]:
score_5 = jiangshan2(score_5)[:,1]
In [41]:
score_5 = jiangshan2(score_5)[:,1]
In [42]:
score_5 = jiangshan2(score_5)[:,1]
In [44]:
count = 1
while count < 20:
score_5 = jiangshan2(score_5)[:,1]
count = count + 1
#¶
In [138]:
test_strptime = test.date.apply(strptime)
In [139]:
test_df = test.copy()
In [140]:
test_df['strptime'] = test_strptime
In [141]:
test_df = test_df.sort_values(by='strptime')
############### 2018-06-04 ########## To be continue ####### By: Li Jiahui¶
#¶
In [121]:
y_label = y_raw[y_raw != -1]
In [373]:
In [375]:
20000 data¶
In [389]:
X_tr_part = X_tr.iloc[0:40000,:]
y_tr_part = y_tr[0:40000]
In [377]:
X_tr_part_imp = X_tr_part.iloc[:,ind]
unlabel data all = 1¶
In [378]:
y_unlabel_1 = y_unlabel.copy()
In [381]:
In [383]:
In [234]:
Out[234]:
semi based on EM¶
First time¶
In [221]:
y_unlabel_2 = rnd_clf.predict(X_unlabel)
In [222]:
y_unlabel_2 = pd.DataFrame(y_unlabel_2)
In [223]:
y2 = pd.concat([y_tr_part, y_unlabel_2]).iloc[:,0]
In [224]:
rnd_clf.fit(X2, y2)
Out[224]:
In [225]:
y_pred_rf = rnd_clf.predict(X_te)
In [226]:
confusion_matrix(y_te, y_pred_rf)
Out[226]:
Second time¶
In [65]:
y_unlabel2 = rnd_clf_1.predict(X_unlabel)
y_unlabel2 = pd.DataFrame(y_unlabel2)
y_tr_2 = pd.concat([y_tr, y_unlabel2])
y_tr_2 = y_tr_2.iloc[:,0]
rnd_clf_2 = RandomForestClassifier(n_estimators=200, max_leaf_nodes=20, n_jobs = -1)
rnd_clf_2.fit(X_tr_add_unlabel, y_tr_2)
Out[65]:
In [70]:
y_pred_rf_2 = rnd_clf_2.predict(X_te)
In [71]:
confusion_matrix(y_te, y_pred_rf_2)
Out[71]:
Third time¶
In [ ]:
y_unlabel = rnd_clf.predict(X_unlabel)
# 合并 label unlabel
X_tr_3
y_tr_3
rnd_clf_3 = RandomForestClassifier(n_estimators=500, max_leaf_nodes=20, n_jobs = -1)
rnd_clf_3.fit(X_tr_3, y_tr_3)
y_pred_rf_3 = rnd_clf_3.predict(X_te)
confusion_matrix(y_te, y_pred_rf_3)
semi based on k-means¶
In [69]:
rnd_clf_2.fit(X_tr_add_unlabel, y_tr_2)
Out[69]:
In [ ]:
sklearn.cluster.KMeans(n_clusters=8, random_state=7, n_jobs= -1 )
semi based on graph regularization¶
In [98]:
a = [1, None, 2]
In [101]:
sum(a,skipna = True)